<?php
/**
* 求职信息采集
* 2012-02-06
* Jken
*/
defined("APP_ROOT") || define("APP_ROOT",dirname(dirname(__FILE__)));
defined("_D_") || define("_D_",DIRECTORY_SEPARATOR);
defined('SCRIPT_NAME') || die("no defined SCRIPT_NAME");

class caiji{
	protected $caijiCore;
	protected $scws;
	protected $fileParse;
	protected $listDir;
	protected $listFile;
	protected $webData;
	protected $callBackCookieFn;

	public function __construct(){
		$this->callBackCookieFn=false;
		$this->caijiCore=false;
		$this->errorLog=APP_ROOT._D_."jkenCaiji"._D_."logCaiji"._D_."errorAppLog.txt";

		$this->setListDir(APP_ROOT._D_."jkenCaiji"._D_."cache"._D_."listfiles");
		$this->setListFile($this->listDir._D_.str_replace('.php','',SCRIPT_NAME).'.txt');
		$this->initParseFile($this->listFile);

		$this->initCaijiObj();
		$this->initScwsObj();
	}

	//初始化采集器
	protected function initCaijiObj(){
		if($this->caijiCore){
			return ($this->caijiCore);
		}
		require(APP_ROOT._D_."jkenCaiji"._D_."caiji.inc.php");
		$this->caijiCore = new caijiCore();
		$this->caijiCore->cTimeOut=6;
		$this->caijiCore->textTimeOut=20;
		$this->caijiCore->pageCharset = "utf-8";

		return ($this->caijiCore);
	}

	//初始化分词系统
	protected function initScwsObj(){
		if($this->scws){
			return ($this->scws);
		}
		require(APP_ROOT._D_."jkenCaiji"._D_."pscws4"._D_."pscws4.class.php");
		$this->scws = new PSCWS4();
		$this->scws->set_debug(false);
		$this->scws->set_charset("utf8");
		$this->scws->set_dict(APP_ROOT._D_."jkenCaiji"._D_."pscws4"._D_."etc"._D_.'dict.utf8.xdb');
		$this->scws->set_rule(APP_ROOT._D_."jkenCaiji"._D_."pscws4"._D_."etc"._D_.'rules.utf8.ini');

		return ($this->scws);
	}

	protected function initParseFile($file){
		if($this->fileParse){
			return ($this->fileParse);
		}
		require(APP_ROOT._D_."jkenCaiji"._D_."parseFile.php");
		$this->fileParse = new parseFile($file);

		return ($this->fileParse);
	}

	public function setListDir($dir){
		$this->listDir=$dir;
	}

	public function getListDir(){
		return ($this->listDir);
	}

	public function setListFile($file){
		$this->listFile=$file;
	}

	public function getListFile(){
		return ($this->listFile);
	}

	public function setCookieValue($cookieValue){
		$this->caijiCore->cookie=$cookieValue;
	} 

	public function setSleepTime($time){
		$this->caijiCore->sleepTime=$time;
	} 

	/*
		获取当前内存占用情况
	*/
	public function memoryInfo(){
		$curMemory = memory_get_usage(true);
		$peakMemory = memory_get_peak_usage(true);

		$curMemory = $curMemory/1024/1024;
		$peakMemory = $peakMemory/1024/1024;

		$curMemory = number_format($curMemory,2,'.','');
		$peakMemory = number_format($peakMemory,2,'.','');

		$memoryInfo=$curMemory.'mb/'.$peakMemory.'mb';

		return ($memoryInfo);
	}

	protected function memberLogin(){
		$submitUrl="http://id.chinalao.com/default.aspx";

		$memberInfo=array();
		$memberInfo["txtUserName"]="chinalao88";
		$memberInfo["txtPwd"]="chinalao88";
		$memberInfo["__VIEWSTATE"]="/wEPDwUKMjEwMDQ0NjM4NQ9kFgJmD2QWBAIHDxYCHgdWaXNpYmxlaGQCCQ8WAh4FdmFsdWUFG2h0dHA6Ly9yZW5jYWkuY2hpbmFsYW8uY29tL2Rk6Q4M1v5HT7nA3yblKw/CBZ0dSVA=";
		$memberInfo["btnLogin"]="登录";
		$loginedRegExp='#会员登录#';

		$logined = $this->caijiCore->toLogin($submitUrl,$memberInfo,$loginedRegExp);
		
		return ($logined);
	}

	public function pickListInfo($listPageTpl,$endPageReg,$parseColumn,$parseReg,$endDate=false,$cutPre=false,$cutEnd=false){
		if(!stristr($listPageTpl,"{{page}}")){
			$errorInfo = "no {{page}} in function pickListInfo!\t";
			$this->writeLog($errorInfo,$listPageTpl);
			return (false);
		}
		$firstListPage=str_replace('{{page}}',1,$listPageTpl);
		$this->caijiCore->url=$firstListPage;
		$this->caijiCore->pickData();
		$totalPages=$this->caijiCore->getTotalPages($endPageReg);
		$totalPages=intval($totalPages);
		if($totalPages<=0){
			$errorInfo = "totalPages error in function pickListInfo!\t";
			$this->writeLog($errorInfo,$firstListPage);
			return (false);
		}

		for($curPage=1;$curPage<=$totalPages;$curPage++){
			$curPageUrl = str_replace('{{page}}',$curPage,$listPageTpl);
			$this->caijiCore->url=$curPageUrl;
			$this->caijiCore->pickData();
			if($cutPre && $cutEnd){
				$this->caijiCore->data=$this->caijiCore->cutData($cutPre,$cutEnd,'in');
			}
			$tmpData=$this->caijiCore->parseDataAll($parseReg);
			if(!$tmpData){
				$curPage--;
				$errorInfo="this page data error in function pickListInfo!\t";
				$this->writeLog($errorInfo,$curPageUrl);
				continue;
			}

			foreach($tmpData[0] as $index => $row){
				$dataRow=array();
				$matchIndex=0;
				foreach($parseColumn as $ckey => $cvalue){
					$filedName = $ckey;
					if($cvalue){						
						$filedValue = $cvalue;
						$dataRow[$filedName]=$filedValue;
					}else{
						$matchIndex++;
						$filedValue = trim($tmpData[$matchIndex][$index]);
						$dataRow[$filedName]=$filedValue;
					}
					
				}

				//在给定的时间之前的数据不再采集[与数据库中数据是否重复无关]
				if($endDate){
					$fieldName=$endDate['fieldName'];
					$endValue=$endDate['endValue'];
					if(isset($dataRow[$fieldName])){
						$tmpDate=strtotime($dataRow[$fieldName]);
						$endValue=strtotime($endValue);
						if($tmpDate < $endValue){
							return true;
						}
					}
				}

				$lineStr = $this->fileParse->formatData($dataRow);
				$this->fileParse->writeToFile($lineStr);
				$this->printInfo($curPage,$totalPages," -> id:".$gcount++);
			}
		}
	}

	public function pickDetailInfo($fieldArr){
		$fp = fopen($this->listFile,'r');
		$gcount=1;
		while(!feof($fp)){
			$this->printInfo(1,1," -> lineId:".$gcount++);
			$lineData = fgets($fp);
			$lineData = trim($lineData);
			if($lineData==''){
				continue;
			}
			$fieldValue=array();
			foreach($fieldArr as $field){
				$tmpValue = $this->fileParse->getFieldData($lineData , $field);
				$fieldValue[$field]=$tmpValue;
			}
			
			$pageurl = $this->fileParse->getFieldData($lineData , 'pageurl');
			$this->caijiCore->url=$pageurl;
			$this->caijiCore->pickData();

			$this->handleDetailData($fieldValue , $this->caijiCore->data);
		}

		fclose($fp);
	}

	protected function handleDetailData($fieldValue , $tmpData){
		
	}

	private function autoDetailInfo($fieldReg,$curPre=false,$curEnd=false){
		$fp = fopen($this->listFile,'r');
		while(!feof($fp)){
			$lineData = fgets($fp);
			//$lineData = trim($lineData,"\r\n");
			$lineData = trim($lineData);
			if($lineData==''){
				continue;
			}			
			$title = $this->fileParse->getFieldData($lineData , 'title');
			$pageurl = $this->fileParse->getFieldData($lineData , 'pageurl');

			$this->caijiCore->url=$pageurl;
			$this->caijiCore->pickData();
			$this->autoHandleDetail($title , $this->caijiCore->data,$fieldReg,$curPre,$curEnd);
		}

		fclose($fp);
	}

	protected function autoHandleDetail($title,$tmpData,$fieldReg,$curPre,$curEnd){
		if($curPre && $curEnd){
			$this->caijiCore->data=$this->caijiCore->cutData($curPre,$curEnd,'in');
		}

		$dataRow=array();
		$dataRow["title"]=$title;
		
		foreach($fieldReg as $fieldKey => $regExp){
			$matches = $this->caijiCore->parseDataOnce($regExp);
			if($matches){
				$dataRow[$fieldKey]=$matches[1];
			}
		}

		return ($dataRow);
	}

	protected function districtInfo($categoryname,$pid=false,&$conn){
		$row = false;
		$sql="select id,categoryname from qs_category_district where 1";
		if($pid!==false){
			$sql .=" and parentid=".$pid;
		}
		$sql .=" and categoryname like '".$categoryname."%'";
		$row = $conn->getone($sql);
		
		return ($row);
	}

	protected function printInfo($curPage,$totalPages,$error){
		$outputInfo = "current page: ".$curPage."/".$totalPages;
		$outputInfo.=" ".$error;
		$outputInfo.=' --- '.$this->memoryInfo();
		$outputInfo.="<br />\n";

		echo $outputInfo;
	}

	protected function writeLog($errorInfo,$url){
		$fp=fopen($this->errorLog,"a+");
		fwrite($fp,date('Y-m-d H:i:s')."\t".$errorInfo."\t".$url."\n");
		fclose($fp);

		echo $errorInfo;
		echo "\n";
		echo "deatils see errorlog please!";
		echo "\n";
	}
}